package com.xiaotu.spider.maoyan.pageprocessor;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.CompanyBasicinfoModel;
import com.xiaotu.common.model.CompanyStarsModel;
import com.xiaotu.common.model.CompanyTvWorksModel;
import com.xiaotu.common.model.MaoYanMovieBasicInfoModel;
import com.xiaotu.common.model.StarBasicinfoModel;
import com.xiaotu.common.model.StarTvWorksModel;
import com.xiaotu.common.model.TVDataSourceRelatetionModel;
import com.xiaotu.common.util.CacheHandler;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.common.util.UUIDUtils;
import com.xiaotu.server.service.maoyan.MaoYanMovieBasicInfoService;
import com.xiaotu.server.service.relation.TVDataSourceRelationService;
import com.xiaotu.spider.AUrlPageProcessor;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;
@Component("MaoYanMovieBasicInfoPageProcessor")
public class MaoYanMovieBasicInfoPageProcessor extends AUrlPageProcessor {
	
	protected Integer tvType = 2;//剧类型 :电影
	protected String resourceSiteId = Constants.SiteName.MAOYAN;//数据来源
	
	private String [] LIST_URL = {
			"http://piaofang.maoyan.com/second-box?beginDate=%s",
		    "http://piaofang.maoyan.com/second-box?beginDate=%s"
	};
	
	@Autowired
	private CacheHandler cacheHandler;
	
	private String todayDate = DateUtils.getDateString(new Date(), DateUtils
            .FILE_NOMENCLATURE_FORMAT);
    
    private String yesterdayDate = DateUtils.getSpecifiedDayBefore(todayDate,DateUtils
            .FILE_NOMENCLATURE_FORMAT);
    
    //全局去重艺人
    private static String SPIDER_DISTINCT_STAR_INFO = "SPIDER_DISTINCT_STAR_INFO_"+DateUtils.getCurrentDate(new Date(), DateUtils.FILE_NOMENCLATURE_FORMAT);
    
    //全局去重公司
    private static String SPIDER_DISTINCT_COMPANY_INFO = "SPIDER_DISTINCT_COMPANY_INFO_"+DateUtils.getCurrentDate(new Date(), DateUtils.FILE_NOMENCLATURE_FORMAT);
    
    //去重网大和电影中的艺人
    private static String STAR_INFO_SUPPLEMENT = "STAR_INFO_SUPPLEMENT_"+DateUtils.getCurrentDate(new Date(), DateUtils.FILE_NOMENCLATURE_FORMAT);
    
    //去重网大和电影中的公司
    private static String COMPANY_INFO_SUPPLEMENT = "COMPANY_INFO_SUPPLEMENT_"+DateUtils.getCurrentDate(new Date(), DateUtils.FILE_NOMENCLATURE_FORMAT);
    
    @Autowired
    private MaoYanMovieBasicInfoService maoYanMovieBasicInfoService;
    
    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("todayDate") != null)
            this.todayDate = map.get("todayDate").toString();
    }
    
    
	@Override
	public void process(Page page) {
		
		Map<String, Object> extrasMap = page.getRequest().getExtras();
		
		switch (this.getPageDataType(page))
        {
            case Constants.PageDataType.BASICINFO:
                this.basicinfoProcess(page,extrasMap);
                break;
            case "_v_=yes":
            	this.basicinfoOtherProcess(page,extrasMap);
            	break;
            case "/companylist":
	            this.basicinfoCompanylistProcess(page,extrasMap);
	            break;
            case "/celebritylist":
            	this.basicinfoPeoplelistProcess(page,extrasMap);
            	break;
            case "/celebrity?id":
            	this.starBasicinfoProcess(page,extrasMap);
            	break;
            case "/companydetail":
            	this.companyBasicinfoProcess(page,extrasMap);
            	break;
            case "/artist":
            	this.companySignActorProcess(page,extrasMap);//签约艺人
            	break;	
            default:
                this.addRequestProcess(page);
                break;
        }
		
	}
	
	//公司签约艺人
	protected void companySignActorProcess(Page page, Map<String, Object> extrasMap) {
		
		String url = page.getRequest().getUrl();
		
		String companyId = extrasMap.get("companyId").toString();
		
		CompanyBasicinfoModel companyBasicinfoModel = (CompanyBasicinfoModel)extrasMap.get("companyBasicinfoModel");
		
		//签约艺人
		String [] signActor = null;
		
		//签约艺人id
		String [] signActorIds = null;

		//签约艺人职业
		String occupationArr = null;
		
		if(url.contains(companyId)){
			
			
			List<CompanyStarsModel> companyStarsList = new ArrayList<CompanyStarsModel>(); 
			
			List<StarTvWorksModel> starTvWorksList = new ArrayList<StarTvWorksModel>();
			
			List<String> panelCList = page.getHtml().xpath("//div[@class='sticky-container']//div[@class='panel-content']//div[@class='panel-c']").all();
			
			if(null != panelCList && panelCList.size()>0 ){
				signActor = new String[panelCList.size()];
				signActorIds = new String[panelCList.size()];
				for(int i=0;i<panelCList.size();i++){
					Html panelCHtml = new Html(panelCList.get(i));
					signActor[i] = panelCHtml.xpath("//div[@class='p-item']//a[@class='p-link']//p[@class='p-item-name ellipsis-1']//text()").get().trim(); 
					signActorIds[i] = panelCHtml.xpath("//div[@class='p-item']//a[@class='p-link']/@data-id").get();
					
					StarTvWorksModel paramObj =  new StarTvWorksModel();
					paramObj.setStar_dataid(signActorIds[i]);
					starTvWorksList.add(paramObj);
					
					String occupation = panelCHtml.xpath("//div[@class='p-item']//a[@class='p-link']//p[@class='p-item-play ellipsis-1']//text()").get();
					
					if(null != occupation && StringUtils.isNotBlank(occupation)){
						occupationArr = occupation.replaceAll("[\\s+]", "").replaceAll("[|]", ",");
						//occupationArr = occupation.split("#");
					}
					
					//将签约艺人与公司成员放入公司艺人中间表缓存中  并且爬取艺人基本信息
					companyStarsList.add(new CompanyStarsModel(
							UUIDUtils.getId(), null, 
							companyId, companyBasicinfoModel.getCompanyname(), 
							null,signActorIds[i], 
							signActor[i], occupationArr, 
							resourceSiteId, System.currentTimeMillis()
							)); 
				}
				
			}
			
			//存在公司成员
			String[] companyPeoplesArr = companyBasicinfoModel.getCompanypeoples();
			String[] companyPeopleidsArr = companyBasicinfoModel.getCompanypeopleids();
			String[] companyPeoplesroleArr = companyBasicinfoModel.getCompanypeoplesrole();
			
			 
			if(null != companyPeoplesArr && companyPeoplesArr.length>0
					
					&& companyPeoplesArr.length == companyPeopleidsArr.length && 
					
					companyPeopleidsArr.length == companyPeoplesroleArr.length){
				
				for(int i =0;i<companyPeoplesArr.length;i++){
					StarTvWorksModel paramObj =  new StarTvWorksModel();
					paramObj.setStar_dataid(companyPeopleidsArr[i]);
					starTvWorksList.add(paramObj);
					companyStarsList.add(new CompanyStarsModel(
							UUIDUtils.getId(), null, 
							companyId, companyBasicinfoModel.getCompanyname(), 
							null,companyPeopleidsArr[i], 
							companyPeoplesArr[i], 
							(companyPeoplesroleArr[i] == null || StringUtils.isBlank(companyPeoplesroleArr[i]))?null:
								companyPeoplesroleArr[i].replaceAll("[\\s+]", "").replaceAll("[|/,，]", ","), 
							resourceSiteId, System.currentTimeMillis()
							)); 
				}
				
			}
			
			companyBasicinfoModel.setSignactor(signActor);
			companyBasicinfoModel.setSignactorids(signActorIds);
			
			//保存
			page.putField(CompanyStarsModel.class.getName(), companyStarsList);
			
			//检查艺人是否存在艺人基础表中  并添加请求爬取艺人基本信息
			checkExitsStarInfo(page,starTvWorksList);
			
			cacheHandler.hSet(COMPANY_INFO_SUPPLEMENT, companyBasicinfoModel.getCompany_dataid(), companyBasicinfoModel);
			cacheHandler.setCacheTimeout(COMPANY_INFO_SUPPLEMENT);
		}
		
	}


	//公司基本信息
	protected void companyBasicinfoProcess(Page page, Map<String, Object> extrasMap) {
		
		String url = page.getRequest().getUrl();
		
		String companyId = extrasMap.get("companyId").toString();
		
		if(url.contains(companyId)){
			
			//公司图片链接
			String imageUrl = null;
			
			//公司名称
			String companyName = null;
			
			//公司英文名
			String companyEngName = null;
			
			//公司简介
			String introduction = null;
			
			//主要作品
			String [] works = null;
			//主要作品对应的id
			String [] workIds = null;
			
			//公司成员
			String [] companyPeoples = null;
			
			//公司成员角色
			String [] companyPeoplesRole = null;
			
			//公司成员id
			String [] companyPeopleIds = null;
			
						
			//公司性质
			String companyProperty = null;
			
			//公司类型
			String [] companyTypes = null;
			
			//子公司
			String [] childCompanys = null;
			
			//子公司id数组
			String [] childCompanyIds = null;
			
			//母公司名称数组
			String [] parentCompanys = null;
			
			//母公司id数组
			String [] parentCompanyIds = null;
			
			//公司地点
			String address = null;
			
			
			//公司图片
			if(null != page.getHtml().xpath("//section[@class='summary-section']//div[@class='chd-logo']//img/@src")){
				imageUrl = page.getHtml().xpath("//section[@class='summary-section']//div[@class='chd-logo']//img/@src").get();
			}
			
			//公司名称
			if(null != page.getHtml().xpath("//section[@class='summary-section']"
					+ "//div[@class='chd-name-holder']"
					+ "//p[@class='chd-name ellipsis-1']"
					+ "//text()").get()){
				companyName = page.getHtml().xpath("//section[@class='summary-section']"
						+ "//div[@class='chd-name-holder']"
						+ "//p[@class='chd-name ellipsis-1']"
						+ "//text()").get();
			}
			//公司英文名
			if(null != page.getHtml().xpath("//section[@class='summary-section']"
					+ "//div[@class='chd-name-holder']"
					+ "//p[@class='chd-ename ellipsis-1']"
					+ "//text()").get()){
				companyEngName = page.getHtml().xpath("//section[@class='summary-section']"
						+ "//div[@class='chd-name-holder']"
						+ "//p[@class='chd-ename ellipsis-1']"
						+ "//text()").get();
				if(StringUtils.isBlank(companyEngName)){
					companyEngName = null;
				}
			}
			//公司简介
			if(null != page.getHtml().xpath("//section[@class='summary-section']"
					+ "//div[@class='summary-intro']"
					+ "//div[@class='expand-content']//text()").get()){
				introduction = page.getHtml().xpath("//section[@class='summary-section']"
						+ "//div[@class='summary-intro']"
						+ "//div[@class='expand-content']//text()").get();
				if(StringUtils.isBlank(introduction)){
					introduction = null;
				}
			}
			
			//主要作品及作品对应的网站剧id
			List<String> workItemList = page.getHtml().xpath("//section[@class='work-section']//a[@class='work-item']").all();
			if(null != workItemList
					&& workItemList.size()>0){
				works = new String[workItemList.size()];
				workIds = new String[workItemList.size()];
				for(int i=0;i<workItemList.size();i++){
					Html workItemHtml = new Html(workItemList.get(i));
					works[i] = workItemHtml.xpath("//p[@class='wi-name ellipsis-1']//text()").get().trim();
					workIds[i] = workItemHtml.xpath("//a[@class='work-item']/@href").get().replaceAll("[^0-9]", "");
				}
			}
			
			//公司成员及角色及id
			
			if(null!=page.getHtml().xpath("//section[@class='staff-section']//a[@class='staff-item']").all()
					&& page.getHtml().xpath("//section[@class='staff-section']//a[@class='staff-item']").all().size()>0){
				List<String> aTagList = page.getHtml().xpath("//section[@class='staff-section']//a[@class='staff-item']").all();
				String[] propleArr = new String[aTagList.size()];
				String[] roleArr = new String[aTagList.size()];
				companyPeopleIds = new String[aTagList.size()];
				for(int i=0; i<aTagList.size();i++){
					Html aTagHtml = new Html(aTagList.get(i));
					if(null != aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-name ellipsis-1']")){
						if(StringUtils.isNotBlank(aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-name ellipsis-1']//text()").get())){
							propleArr[i] = aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-name ellipsis-1']//text()").get();
						}else{
							propleArr[i] = "";
						}
						if(StringUtils.isNotBlank(aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-role ellipsis-1']//text()").get())){
							roleArr[i] = aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-role ellipsis-1']//text()").get();
						}else{
							roleArr[i] = "";
						}
					}
					companyPeopleIds[i] = aTagHtml.xpath("//a[@class='staff-item']/@href").get().replaceAll("[^0-9]", "");
				}
				companyPeoples = propleArr;
				companyPeoplesRole = roleArr;
				
			}
			
			
			//公司性质
			if(null != page.getHtml().xpath("//section[@class='info-section']"
					+ "//div[@class='expandable theme-text']"
					+ "//div[@class='expand-content']"
					+ "//div[@class='i-item i-nature']"
					+ "//p[@class='i-label']//text()").get()){
				if(page.getHtml().xpath("//section[@class='info-section']"
						+ "//div[@class='expandable theme-text']"
						+ "//div[@class='expand-content']"
						+ "//div[@class='i-item i-nature']"
						+ "//p[@class='i-label']//text()").get().contains("公司性质")){
					companyProperty = page.getHtml().xpath("//section[@class='info-section']"
							+ "//div[@class='expandable theme-text']"
							+ "//div[@class='expand-content']"
							+ "//div[@class='i-item i-nature']"
							+ "//p[@class='i-val ellipsis-1']//text()").get().trim();
					if(StringUtils.isBlank(companyProperty)){
						companyProperty = null;
					}
				}
			}
			
			//公司类型
			if(null != page.getHtml().xpath("//section[@class='info-section']"
					+ "//div[@class='expandable theme-text']"
					+ "//div[@class='expand-content']"
					+ "//div[@class='i-item i-ctype']"
					+ "//p[@class='i-label']//text()").get()){
				if(page.getHtml().xpath("//section[@class='info-section']"
						+ "//div[@class='expandable theme-text']"
						+ "//div[@class='expand-content']"
						+ "//div[@class='i-item i-ctype']"
						+ "//p[@class='i-label']//text()").get().contains("公司类型")){
					companyTypes = page.getHtml().xpath("//section[@class='info-section']"
							+ "//div[@class='expandable theme-text']"
							+ "//div[@class='expand-content']"
							+ "//div[@class='i-item i-ctype']"
							+ "//p[@class='i-val ellipsis-2']//text()").get().replaceAll("[ ]", ",").split(",");
					if(companyTypes.length==0){
						companyTypes = null;
					}
				}
			}
			
			//子公司及子公司id
			List<String> childCompanyList = page.getHtml().xpath("//section[@class='info-section']"
					+ "//div[@class='i-item i-child']"
					+ "//a[@class='i-val ellipsis-1']").all();
			if(null != childCompanyList && childCompanyList.size()>0){
				childCompanys = new String[childCompanyList.size()];
				childCompanyIds = new String[childCompanyList.size()];
				for(int i=0;i<childCompanyList.size();i++){
					Html childCompanyHtml = new Html(childCompanyList.get(i));
					childCompanys[i] = childCompanyHtml.xpath("//span//text()").get().trim();
					childCompanyIds[i] = childCompanyHtml.xpath("//a[@class='i-val ellipsis-1']/@href").get().replaceAll("[^0-9]", "");
				}
			}
			
			//母公司及公司id
			List<String> parentCompanyList = page.getHtml().xpath("//section[@class='info-section']"
					+ "//div[@class='i-item i-parent']"
					+ "//a[@class='i-val ellipsis-1']").all();
			if(null != parentCompanyList && parentCompanyList.size()>0){
				parentCompanys = new String[parentCompanyList.size()];
				parentCompanyIds = new String[parentCompanyList.size()];
				for(int i=0;i<parentCompanyList.size();i++){
					Html parentCompanyHtml = new Html(parentCompanyList.get(i));
					parentCompanys[i] = parentCompanyHtml.xpath("//span//text()").get().trim();
					parentCompanyIds[i] = parentCompanyHtml.xpath("//a[@class='i-val ellipsis-1']/@href").get().replaceAll("[^0-9]", "");
				}
			}
			
			
			//公司地点
			if(null!= page.getHtml().xpath("//section[@class='contact-section']//div[@class='contact']//p[@class='c-address']").get()){
				address = page.getHtml().xpath("//section[@class='contact-section']//div[@class='contact']//p[@class='c-address']//text()").get();
				if(StringUtils.isBlank(address)){
					address = null;
				}
			}
			
			//保存公司的基本信息
			CompanyBasicinfoModel companyBasicinfoModel = new CompanyBasicinfoModel(
					UUIDUtils.getId(),null, 
					companyId, Constants.SiteName.MAOYAN,
					companyName, companyEngName,
					companyProperty, companyTypes,
				    works,workIds, 
				    parentCompanys, parentCompanyIds,
				    childCompanys,childCompanyIds, 
				    address, introduction,
				    companyPeoples,companyPeoplesRole,
				    companyPeopleIds,null, 
				    null,imageUrl, 
				    System.currentTimeMillis()
					);
			
			extrasMap.put("companyBasicinfoModel", companyBasicinfoModel);
			
			//用于提取签约艺人
			PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/companydetail/%s/artist", companyId), extrasMap);
			
		}
	}


	//艺人基本信息
	protected void starBasicinfoProcess(Page page,Map<String, Object> extrasMap) {
		
		String url = page.getRequest().getUrl();
		
		String starId = extrasMap.get("starId").toString();
		
		if(url.contains(starId)){
			//艺人图片链接
			String imageUrl = page.getHtml().xpath("//div[@class='celebrity-header cele-main']"
					+ "//div[@class='celebrity-bg']"
					+ "//img[@class='sm-big-img']"
					+ "/@src").get();
			//名字
			String starName = null;
			//英文名
			String engName = null;
			List<String> list = page.getHtml().xpath("//div[@class='celebrity-header cele-main']"
					+ "//div[@class='celebrity-bg']"
					+ "//div[@class='celebrity-profile']"
					+ "//div[@class='string-tip']//p//text()").all();
			if(null != list && list.size()>0){
				for(int i =0;i<list.size();i++){
					if(i==0){
						starName = list.get(i).trim();
					}else if(i==1){
						engName = list.get(i).trim();
					}
				}
				
			}
			
			//职业
			String [] occupation = null;
			List<String> tempList = new ArrayList<String>();
			List<String> occupationList = page.getHtml().xpath("//div[@class='celebrity-header cele-main']"
					+ "//div[@class='celebrity-bg']"
					+ "//div[@class='celebrity-profile']"
					+ "//div[@class='common-area']"
					+ "//div[@class='role-name-list string-ellipsis string-show-all']//span").all();
			if(null!=occupationList && occupationList.size()>0){
				occupation = new String[occupationList.size()];
				for(int i = 0;i<occupationList.size();i++){
					String occName = new Html(occupationList.get(i)).xpath("//span/@class").toString();
					String occValue = new Html(occupationList.get(i)).xpath("//span//text()").toString();
					occupation[i] = occupationList.get(i).trim();
					tempList.add(occValue);
					if(occName.contains("last-child")) {
						break;
					}
				}
				occupation = tempList.toArray(new String[tempList.size()]);
			}
			
			//国籍
			String country = null;
			//民族
			String nation = null;
			//性别
			String sex = null;
			//血型
			String bloodType = null;
			//身高
			String height = null;
			//体重
			String weight = null;
			
			
			List<String> contentPageList = page.getHtml().xpath("//div[@class='celeAnchor-container body-container']"
					+ "//div[@class='cele-main-content-area']"
					+ "//div[@class='cele-main cele-desc cele-basic-info expandable']"
					+ "//div[@class='content-page-block']"
					+ "//p[@class='content-page']").all();
			
			for(String content : contentPageList){
				if(null != new Html(content).xpath("//p[@class='content-page']//span").all()){
					if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("国籍")){
						country = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("民族")){
						nation = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("性别")){
						sex = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("身高")){
						height = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("血型")){
						bloodType = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("体重")){
						weight = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}
				};
			}
			//出生日期
			String birthday = null;
			//毕业院校
			String school = null;
			//别名
			String [] alias = null;
			//出生地
			String born = null;
			//简介
			String introduction = null;
			
			List<String> divList = page.getHtml().xpath("//div[@class='content-wrapper']//div[@class='cele-introduction expand-content content-page']//div").all();
			if(null != divList && divList.size()>0){
				for(String divStr : divList){
					Html divHtml = new Html(divStr);
					if(divHtml.xpath("//div/@data-id").toString().contains("birthday")){//1958-07-14（巨蟹座）
						birthday = divHtml.xpath("//div//span//text()").all().get(1).trim().replaceAll("[\u4e00-\u9fa5（）()]", "");
					}else if(divHtml.xpath("//div/@data-id").toString().contains("school")){
						school = divHtml.xpath("//div//span//text()").all().get(1).trim();
					}else if(divHtml.xpath("//div/@data-id").toString().contains("aliasName")){
						alias = divHtml.xpath("//div//span//text()").all().get(1).trim().replaceAll("[\"]", "").replaceAll("[,，/]", ",").split(",");
					}else if(divHtml.xpath("//div/@data-id").toString().contains("birthPlace")){
						born = divHtml.xpath("//div//span//text()").all().get(1).trim();
					}else if(divHtml.xpath("//div/@data-id").toString().contains("intro")){
						introduction = divHtml.xpath("//div//span//text()").all().get(1).trim();
					}
				}
			}
			
			
			StarBasicinfoModel starBasicinfoModel = new StarBasicinfoModel(
					UUIDUtils.getId(),null,starId,Constants.SiteName.MAOYAN,
					starName,engName,occupation,country,
					alias,sex,height,weight,
					bloodType,nation,school,
					born,birthday,introduction,
					imageUrl,System.currentTimeMillis()
					);
			
			cacheHandler.hSet(STAR_INFO_SUPPLEMENT, starId, starBasicinfoModel);
			cacheHandler.setCacheTimeout(STAR_INFO_SUPPLEMENT);
		}
		
	}


	//人员相关
	@SuppressWarnings("unchecked")
	protected void basicinfoPeoplelistProcess(Page page, Map<String, Object> extrasMap) {
		String url = page.getRequest().getUrl();
		String dataId = extrasMap.get("dataId").toString();
		if(url.contains(dataId)){
			
			//存放艺人与作品
			List<StarTvWorksModel> starTvWorksModelList = new ArrayList<StarTvWorksModel>();
			
			MaoYanMovieBasicInfoModel maoYanMovieBasicInfoModel = (MaoYanMovieBasicInfoModel) extrasMap.get("movieBasicInfo");
			
			List<String> dlTagAll = page.getHtml().xpath("//div[@class='sticky-container']//div[@class='scroller cat-wrapper']//div[@class='wrapper cat-content']"
					+ "//div[@class='panel-wrapper']//dl").all();
			for(String dl : dlTagAll){
				
				Map<String,Object> result = convertPeopleRelateInfo(new Html(dl),false,maoYanMovieBasicInfoModel,null);
				//获取所有人员
				Map<String,Object> resultAll = convertPeopleRelateInfo(new Html(dl),true,maoYanMovieBasicInfoModel,starTvWorksModelList);
				starTvWorksModelList = (List<StarTvWorksModel>)resultAll.get("starTvWorksModelList"); 
				
				
				if(result.get("flag").equals("导演")){
					String [] nameArr = ((String [][])result.get("nameAndEngNameAndRole"))[0];
					String [] engNameArr = ((String [][])result.get("nameAndEngNameAndRole"))[1];
					maoYanMovieBasicInfoModel.setDirector(nameArr);
					maoYanMovieBasicInfoModel.setDirectorEnglishName(engNameArr);
				}
				else if(result.get("flag").equals("演员")){
					String [] nameArr = ((String [][])result.get("nameAndEngNameAndRole"))[0];
					String [] engNameArr = ((String [][])result.get("nameAndEngNameAndRole"))[1];
					String [] roleArr = ((String [][])result.get("nameAndEngNameAndRole"))[2];
					maoYanMovieBasicInfoModel.setActor(nameArr);
					maoYanMovieBasicInfoModel.setActorEnglishName(engNameArr);
					maoYanMovieBasicInfoModel.setActorRole(roleArr);
				}
				else if(result.get("flag").equals("编剧")){
					String [] nameArr = ((String [][])result.get("nameAndEngNameAndRole"))[0];
					String [] engNameArr = ((String [][])result.get("nameAndEngNameAndRole"))[1];
					maoYanMovieBasicInfoModel.setScreenWriter(nameArr);
					maoYanMovieBasicInfoModel.setScreenwriterEnglishName(engNameArr);
				}
				else if(result.get("flag").equals("制片人")){
					String [] nameArr = ((String [][])result.get("nameAndEngNameAndRole"))[0];
					String [] engNameArr = ((String [][])result.get("nameAndEngNameAndRole"))[1];
					maoYanMovieBasicInfoModel.setProducer(nameArr);
					maoYanMovieBasicInfoModel.setProducerEnglishName(engNameArr);
				}
				else if(result.get("flag").equals("摄影师")){
					String [] nameArr = ((String [][])result.get("nameAndEngNameAndRole"))[0];
					String [] engNameArr = ((String [][])result.get("nameAndEngNameAndRole"))[1];
					maoYanMovieBasicInfoModel.setPhotographer(nameArr);
					maoYanMovieBasicInfoModel.setPhotographerEnglishName(engNameArr);
				}
				else if(result.get("flag").equals("出品人")){
					String [] nameArr = ((String [][])result.get("nameAndEngNameAndRole"))[0];
					String [] engNameArr = ((String [][])result.get("nameAndEngNameAndRole"))[1];
					maoYanMovieBasicInfoModel.setPublisher(nameArr);
					maoYanMovieBasicInfoModel.setPublisherEnglishName(engNameArr);
				}
				else if(result.get("flag").equals("发行")){
					String [] nameArr = ((String [][])result.get("nameAndEngNameAndRole"))[0];
					String [] engNameArr = ((String [][])result.get("nameAndEngNameAndRole"))[1];
					maoYanMovieBasicInfoModel.setIssuer(nameArr);
					maoYanMovieBasicInfoModel.setIssuerEnglishName(engNameArr);
				}
			}
			
			//检查艺人是否存在艺人基础表中  并添加请求爬取艺人基本信息
			checkExitsStarInfo(page,starTvWorksModelList);
			//保存
			page.putField(MaoYanMovieBasicInfoModel.class.getName(), maoYanMovieBasicInfoModel);
			page.putField(StarTvWorksModel.class.getName(), starTvWorksModelList);
		}
	}
	
	
	//检查艺人是否存在艺人基础表中
	private void checkExitsStarInfo(Page page,List<StarTvWorksModel> starTvWorksModelList){
		
		List<StarTvWorksModel> list = new ArrayList<StarTvWorksModel>();
		list.addAll(starTvWorksModelList);
		List<StarTvWorksModel> exitsList = maoYanMovieBasicInfoService.queryExitsStarInfoList(list);
		
		for(int i=0;i<list.size();){
			boolean flag = false;
			StarTvWorksModel outModel = list.get(i);
			for(StarTvWorksModel  exitsModel : exitsList){
				if(outModel.getStar_dataid().equals(exitsModel.getStar_dataid())){
					list.remove(i);
					flag = true;
					break;
				}
			}
			if(flag){
				i=0;
			}else{
				i++;
			}
		}
		//去重
		Map<String,String> distinctMap = new HashMap<String,String>();
		for(StarTvWorksModel noExitsModel : list){
			if(!distinctMap.containsKey(noExitsModel.getStar_dataid())){
				distinctMap.put(noExitsModel.getStar_dataid(), noExitsModel.getStarname());
			}
		}
		
		for(Entry<String, String> entry : distinctMap.entrySet()){
			Map<String,Object> extrasMap = new HashMap<String,Object>();
			extrasMap.put("starId", entry.getKey());
			if(!cacheHandler.cacheExist(SPIDER_DISTINCT_STAR_INFO)){
				PageUtils.addTargetRequest(page, "https://piaofang.maoyan.com/celebrity?id="+entry.getKey(), extrasMap);
				cacheHandler.hSet(SPIDER_DISTINCT_STAR_INFO, entry.getKey(), resourceSiteId+"_"+entry.getKey()+"_"+entry.getValue());
			}else{
				String hGetAll = cacheHandler.hGet(SPIDER_DISTINCT_STAR_INFO, entry.getKey(), String.class);
				if(null == hGetAll){
					PageUtils.addTargetRequest(page, "https://piaofang.maoyan.com/celebrity?id="+entry.getKey(), extrasMap);
	  	    		cacheHandler.hSet(SPIDER_DISTINCT_STAR_INFO, entry.getKey(), resourceSiteId+"_"+entry.getKey()+"_"+entry.getValue());
	  	    	}
			}
			
		}
		//设置失效时间
		cacheHandler.setCacheTimeout(SPIDER_DISTINCT_STAR_INFO);
	}
	
	
	
	
	private Map<String,Object> convertPeopleRelateInfo(Html dlHtml,boolean isAll,MaoYanMovieBasicInfoModel maoYanMovieBasicInfoModel,List<StarTvWorksModel> starTvWorksModelList){
		
		
		Map<String,Object> result = new HashMap<String,Object>();
		
		String occupation = dlHtml.xpath("//dl[@class='panel-main category']//dt[@class='panel-header sticky-wrapper']//span[@class='title-name']//text()").get();
		
		List<String> contentAll = dlHtml.xpath("//dl[@class='panel-main category']//dd[@class='panel-content']//div[@class='p-item']").all();
		String[][] nameAndEngNameAndRole = new String[3][];
		int size = 0;
		if(!isAll){
			size = contentAll.size()<=10?contentAll.size():10;
		}else{
			size = contentAll.size();
		}
		String [] nameArr = new String[size]; 
		String [] engNameArr = new String[size];
		String [] roleArr = new String[size];
//		if(occupation.equals("导演") || occupation.equals("演员") || occupation.equals("编剧")
//				|| occupation.equals("制片人") || occupation.equals("摄影师")
//				|| occupation.equals("出品人") || occupation.equals("发行")){
		int ordertype = 1;
		for(int i=0;i<size;i++){
			
			
			String starId = new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']/@data-id").get();
			
			String name = new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
					+ "//p[@class='p-item-name ellipsis-1']//text()").get()==null?"":new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
							+ "//p[@class='p-item-name ellipsis-1']//text()").get();
			String engName = new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
					+ "//p[@class='p-item-e-name ellipsis-1']//text()").get()==null?"":new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
							+ "//p[@class='p-item-e-name ellipsis-1']//text()").get();
			String role = new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
					+ "//p[@class='p-item-play ellipsis-1']//text()").get()==null?"":new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
							+ "//p[@class='p-item-play ellipsis-1']//text()").get().replace("饰：", "").replace("（配音）", "");
			nameArr[i] = name.replaceAll("[\"]", "").replace(",", "/");
			engNameArr[i] = engName.replaceAll("[\"]", "").replace(",", "/");
			roleArr[i] = role.replaceAll("[\"]", "").replace(",", "/");
			if(isAll){
				starTvWorksModelList.add(new StarTvWorksModel(
						UUIDUtils.getId(),null,null,
						maoYanMovieBasicInfoModel.getDataid(),
						maoYanMovieBasicInfoModel.getTvname(),
						starId, name, role,
						occupation,Constants.SiteName.MAOYAN,
						ordertype++,
						System.currentTimeMillis()
						));
			}
		}
		for(int j=0;j<nameArr.length;j++){
			if(StringUtils.isNotEmpty(nameArr[j])){
				nameAndEngNameAndRole[0] = nameArr;
			}
			if(StringUtils.isNotEmpty(engNameArr[j])){
				nameAndEngNameAndRole[1] = engNameArr;
			}
			if(StringUtils.isNotEmpty(roleArr[j])){
				nameAndEngNameAndRole[2] = roleArr;
			}
		}
			
		//}
		result.put("nameAndEngNameAndRole", nameAndEngNameAndRole);
		result.put("flag", occupation);
		if(null != starTvWorksModelList && isAll){
			result.put("starTvWorksModelList", starTvWorksModelList);
		}
		return result;
	}
	

	//参与的公司
	@SuppressWarnings("unchecked")
	protected void basicinfoCompanylistProcess(Page page, Map<String, Object> extrasMap) {
		String url = page.getRequest().getUrl();
		String dataId = extrasMap.get("dataId").toString();
		if(url.contains(dataId)){
			MaoYanMovieBasicInfoModel maoYanMovieBasicInfoModel = (MaoYanMovieBasicInfoModel) extrasMap.get("movieBasicInfo");
			//制作公司
			String [] productionCompanyArr = null;
			//联合出品公司
			String[] jointIssueCompanyArr = null;
			//出品公司
			String [] issueCompanyArr = null;
			//发行公司
			String[] publishCompanyArr = null;
			//联合发行公司
			String [] jointPublishCompanyArr= null;
			//宣传公司
			String [] propagandaCompanyArr = null;
			//营销公司
			String [] marketCompanyArr = null;
			//海报制作公司
			String [] posterProductionCompanyArr = null;
			//预告片制作公司
		    String [] trailerProductionCompanyArr = null;
			//其他公司
			String[] otherCompanyArr = null;
			
			Html html = page.getHtml();
			
			try {
				List<String> all = html.xpath("//div[@class='wrapper cat-content']//div[@class='panel-wrapper']//dl").all();
				List<CompanyTvWorksModel> companyTvWorksModelList = new ArrayList<CompanyTvWorksModel>();
				for(String dlStr : all){
					Html dlHtml = new Html(dlStr);
					String companyNameStr = dlHtml.xpath("//dl[@class='panel-main category']//span[@class='title-left']//span[@class='title-name']//text()").get();
					if(StringUtils.isNotBlank(companyNameStr)){
						String currCompanyNameStr = companyNameStr;
						if(companyNameStr.contains("其他")){
							currCompanyNameStr = "";
						}
						companyTvWorksModelList = (List<CompanyTvWorksModel>)this.getCompanyInfoList(maoYanMovieBasicInfoModel,dlHtml,currCompanyNameStr,productionCompanyArr,true,companyTvWorksModelList).get("companyTvWorksModelList");
						
						switch (companyNameStr){
							case "制作":
								productionCompanyArr = (String [])this.getCompanyInfoList(null,dlHtml,"need_remove_tail_brackets",productionCompanyArr,false,null).get("tempArr");
							break;
							case "出品":
								issueCompanyArr = (String [])this.getCompanyInfoList(null,dlHtml,"need_remove_tail_brackets",issueCompanyArr,false,null).get("tempArr");
							break;
							case "联合出品":
								jointIssueCompanyArr = (String [])this.getCompanyInfoList(null,dlHtml,"need_remove_tail_brackets",jointIssueCompanyArr,false,null).get("tempArr");
							break;
							case "发行":
								publishCompanyArr = (String [])this.getCompanyInfoList(null,dlHtml,"need_remove_tail_brackets",publishCompanyArr,false,null).get("tempArr");
							break;
							case "联合发行":
								jointPublishCompanyArr = (String [])this.getCompanyInfoList(null,dlHtml,"need_remove_tail_brackets",jointPublishCompanyArr,false,null).get("tempArr");
							break;
							case "其他":
								otherCompanyArr = (String [])this.getCompanyInfoList(null,dlHtml,"",otherCompanyArr,false,null).get("tempArr");
								if(null != otherCompanyArr && otherCompanyArr.length>0){
									List<String> propagandaList = new ArrayList<String>();//宣传
									List<String> marketList = new ArrayList<String>();//营销
									List<String> posterList = new ArrayList<String>();//海报
									List<String> trailerList = new ArrayList<String>();//预告片
									List<String> otherList = new ArrayList<String>();//其他
									for(String companyName : otherCompanyArr){
										if(companyName.indexOf("（")!=-1){
											String tempCompanyName = companyName.substring(companyName.indexOf("（"));
											if(tempCompanyName.contains("宣传")){
												propagandaList.add(companyName);
											}else if(tempCompanyName.contains("营销")){
												marketList.add(companyName);
											}else if(tempCompanyName.contains("预告片")){
												trailerList.add(companyName);
											}else if(tempCompanyName.contains("海报")){
												posterList.add(companyName);
											}else{
												otherList.add(companyName);
											}
										}
									}
									propagandaCompanyArr = (String[])convertListToArray(null,propagandaList,null,"need_remove_tail_brackets",false,null).get("tempArr");
									marketCompanyArr = (String[])convertListToArray(null,marketList,null,"need_remove_tail_brackets",false,null).get("tempArr"); 
									posterProductionCompanyArr = (String[])convertListToArray(null,posterList,null,"need_remove_tail_brackets",false,null).get("tempArr"); 
									trailerProductionCompanyArr = (String[])convertListToArray(null,trailerList,null,"need_remove_tail_brackets",false,null).get("tempArr"); 
									otherCompanyArr = (String[])convertListToArray(null,otherList,null,"need_remove_tail_brackets",false,null).get("tempArr"); 
								}
							break;
						}
					}
				}
				//保存公司代表作
				page.putField(CompanyTvWorksModel.class.getName(), companyTvWorksModelList);
				
				//检查公司是否存在表中
				checkExitsCompanyInfo(page,companyTvWorksModelList);
				
			} catch (Exception e) {
				System.out.println("异常信息 "+MaoYanMovieBasicInfoPageProcessor.class.getName()+" methodName : basicinfoCompanylistProcess "+e.getMessage());
			}
			maoYanMovieBasicInfoModel.setProductionCompany(productionCompanyArr);
			maoYanMovieBasicInfoModel.setIssueCompany(issueCompanyArr);
			maoYanMovieBasicInfoModel.setJointIssueCompany(jointIssueCompanyArr);
			maoYanMovieBasicInfoModel.setPublishCompany(publishCompanyArr);
			maoYanMovieBasicInfoModel.setJointPublishCompany(jointPublishCompanyArr);
			maoYanMovieBasicInfoModel.setPropagandaCompany(propagandaCompanyArr);
			maoYanMovieBasicInfoModel.setMarketCompany(marketCompanyArr);
			maoYanMovieBasicInfoModel.setPosterProductionCompany(posterProductionCompanyArr);
			maoYanMovieBasicInfoModel.setTrailerProductionCompany(trailerProductionCompanyArr);
			maoYanMovieBasicInfoModel.setOtherCompany(otherCompanyArr);
			extrasMap.put("movieBasicInfo", maoYanMovieBasicInfoModel);
			PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/movie/%s/celebritylist", dataId), extrasMap);
		}
		
	}

	//检查公司是否存在表中
	private void checkExitsCompanyInfo(Page page, List<CompanyTvWorksModel> companyTvWorksModelList) {
		
		List<CompanyTvWorksModel> list = new ArrayList<CompanyTvWorksModel>();
		list.addAll(companyTvWorksModelList);
		List<CompanyTvWorksModel> exitsList = maoYanMovieBasicInfoService.queryExitsCompanysInfoList(list);
		for(int i=0;i<list.size();){
			boolean flag = false;
			CompanyTvWorksModel outModel = list.get(i);
			for(CompanyTvWorksModel  exitsModel : exitsList){
				if(outModel.getCompany_dataid().equals(exitsModel.getCompany_dataid())){
					list.remove(i);
					flag = true;
					break;
				}
			}
			if(flag){
				i=0;
			}else{
				i++;
			}
		}
		//去重
		Map<String,String> distinctMap = new HashMap<String,String>();
		
		for(CompanyTvWorksModel noExitsModel : list){
			if(!distinctMap.containsKey(noExitsModel.getCompany_dataid())){
				distinctMap.put(noExitsModel.getCompany_dataid(), noExitsModel.getCompanyname());
			}
		}
		
		for(Entry<String, String> entry : distinctMap.entrySet()){
			Map<String,Object> extrasMap = new HashMap<String,Object>();
			extrasMap.put("companyId", entry.getKey());
			if(!cacheHandler.cacheExist(SPIDER_DISTINCT_COMPANY_INFO)){
				cacheHandler.hSet(SPIDER_DISTINCT_COMPANY_INFO, entry.getKey(), resourceSiteId+"_"+entry.getKey()+"_"+entry.getValue());
				PageUtils.addTargetRequest(page, "https://piaofang.maoyan.com/companydetail/"+entry.getKey(), extrasMap);
			}else{
				String hGetAll = cacheHandler.hGet(SPIDER_DISTINCT_COMPANY_INFO, entry.getKey(), String.class);
				if(null == hGetAll){
					PageUtils.addTargetRequest(page, "https://piaofang.maoyan.com/companydetail/"+entry.getKey(), extrasMap);
					cacheHandler.hSet(SPIDER_DISTINCT_COMPANY_INFO, entry.getKey(), resourceSiteId+"_"+entry.getKey()+"_"+entry.getValue());
	  	    	}
			}
		}
		//设置失效时间
		cacheHandler.setCacheTimeout(SPIDER_DISTINCT_COMPANY_INFO);
	}


	private Map<String,Object> getCompanyInfoList(MaoYanMovieBasicInfoModel maoYanMovieBasicInfoModel,
			Html html,String companyType,String [] companyArr,
			boolean isAll,List<CompanyTvWorksModel> companyTvWorksModelList){
		List<String> companyList = html.xpath("//dl[@class='panel-main category']//dd[@class='panel-content']"
				+ "//div[@class='panel-c']//div[@class='p-item']"
				+ "//div[@class='p-desc']//p//text()").all();
		List<String> companyIdList = html.xpath("//dl[@class='panel-main category']//dd[@class='panel-content']"
				+ "//div[@class='panel-c']//div[@class='p-item']"
				+ "//div[@class='p-link']/@data-id").all();
		if(companyList.size()>0 && companyIdList.size()>0 && (companyList.size() == companyIdList.size())){
			return convertListToArray(maoYanMovieBasicInfoModel,companyList,companyIdList,companyType,isAll,companyTvWorksModelList);
		}
		return null;
	}
	
	private Map<String,Object> convertListToArray(MaoYanMovieBasicInfoModel maoYanMovieBasicInfoModel,
			List<String> nameList,List<String> idlist,
			String companyType,boolean isAll,List<CompanyTvWorksModel> companyTvWorksModelList){
		Map<String,Object> returnMap = new HashMap<String,Object>();
		String [] tempArr = new String[nameList.size()];
		int ordertype = 1;
		for(int i=0;i<nameList.size();i++){
			tempArr[i] = nameList.get(i).trim();
			if(companyType.equals("need_remove_tail_brackets")){//去除尾部括号
				tempArr[i] = StringUtil.removeBrackets(nameList.get(i).trim());
			}
			if(isAll && null!=companyTvWorksModelList){
				CompanyTvWorksModel companyTvWorksModel = new CompanyTvWorksModel(
						UUIDUtils.getId(),null,null,
						maoYanMovieBasicInfoModel.getDataid(),
						maoYanMovieBasicInfoModel.getTvname(),
						idlist.get(i).trim(),
						companyType.equals("")==true?StringUtil.getBracketsContent(tempArr[i])[0]:tempArr[i],
						companyType.equals("")==true?StringUtil.getBracketsContent(tempArr[i])[1]:companyType,
						Constants.SiteName.MAOYAN,
						ordertype++,
						System.currentTimeMillis()
						);
				
				companyTvWorksModelList.add(companyTvWorksModel);
			}
		}
		returnMap.put("companyTvWorksModelList", companyTvWorksModelList);
		returnMap.put("tempArr", tempArr.length==0?null:tempArr);
		return returnMap;
	}
	

	@SuppressWarnings("null")
	protected void basicinfoOtherProcess(Page page, Map<String, Object> extrasMap) {
		MaoYanMovieBasicInfoModel maoYanMovieBasicInfoModel = (MaoYanMovieBasicInfoModel) extrasMap.get("movieBasicInfo");
		String url = page.getRequest().getUrl();
		String dataId = extrasMap.get("dataId").toString();
		if(url.contains(dataId)){
			//英文名
			String englishName = null;
			//图片链接
			String imageUrl = null;
			//类型
			String [] label = null;
			//地区
			String [] area = null;
			//时长
			String timeLength = null;
			//播放类型
			String playType = null;
			//大陆上映地区
			String showingArea = null;
			String[] dateAndArea = new String[2];
			try {
				
				//首先判断是电影（2）还是网大（6）
//				String netMovieFlag = page.getHtml().xpath("//div[@class='sections tv-page']").get();
//				String movieFlag = page.getHtml().xpath("//div[@class='sections']").get();
//				if(null != netMovieFlag){
//					tvType = 6;
//				}else{
//					if(null != movieFlag){
//						tvType = 2;
//					}
//				}
				
				//存放剧的类型
				maoYanMovieBasicInfoModel.setTvType(tvType);
				
				//英文名
				englishName = page.getHtml().xpath("//div[@class='info-detail']//p[@class='info-etitle']//span[@class='info-etitle-content']//text()").get();
				
				
				imageUrl = page.getHtml().xpath("//div[@class='info-base']//div[@class='info-poster']//img/@src").get();
				
				if(null == imageUrl || StringUtils.isBlank(imageUrl)){
					
					imageUrl = page.getHtml().xpath("//div[@class='movie-baseinfo']//div[@class='movie-container']//img/@src").get();
					
				}
				
				if(null == imageUrl || StringUtils.isBlank(imageUrl)){
					
					imageUrl = page.getHtml().xpath("//div[@class='tv-container']//div[@class='info-poster']//img/@src").get();
					
				}
				
				//保存图片链接
				maoYanMovieBasicInfoModel.setImageurl(imageUrl);
				
				
				String labelStr = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//p[@class='info-category']//text()").get();
				//补充类型
				if(null == labelStr){
					if(null != page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all()
							&& page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all().size()>0){
						labelStr = HtmlUtil.delHTMLTag(page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all().get(0));
					}
				}
				
				if(StringUtils.isNotBlank(labelStr)){
					label = labelStr.replaceAll("[/]", ",").replaceAll("[ ]", "").split(",");
					if(label.length == 0){
						label = null;
					}
				}
				
				String arealStr = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//div[@class='info-source-duration']//p[@class='.ellipsis-1']//text()").get();
				if(StringUtils.isNotBlank(arealStr)){
					area = arealStr.replace("/", "").replace("\"", "").trim().split(",");
					if(area.length == 0){
						area = null;
					}
				}
				//时长
				timeLength = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//div[@class='info-source-duration']//span//text()").get();
				
				//补充时长
				if(null == timeLength){
					if(null!= page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all()
							&& page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all().size()==2){
						timeLength = HtmlUtil.delHTMLTag(page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all().get(1));
					}
				}
				
				
				if(StringUtils.isNotBlank(timeLength) && timeLength.indexOf("集数")!=-1){
					//存放剧的类型
					maoYanMovieBasicInfoModel.setTvType(1);//电视剧
				}
				
				
				if(StringUtils.isNotBlank(timeLength) && timeLength.indexOf("集数")==-1){
					timeLength = timeLength.replaceAll("[\u4e00-\u9fa5]", "").trim();
				}else{
					timeLength = null;
				}
				
				//播放类型
				playType = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//span[@class='info-tag']//text()").get();
				//大陆上映地区
				showingArea = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//div[@class='info-release']//span[@class='score-info ellipsis-1']//text()").get();
				
				if(null == showingArea){
					showingArea = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
							+ "/div[@class='detail-list-content']//div[@class='info-source-duration']//p//text()").get();
				}
				
				String showingDate = null;
				if(null == showingArea){
					if(null != page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='info-list']//p").all()
							&& page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='info-list']//p").all().size()==3){
						showingDate = HtmlUtil.delHTMLTag(page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='info-list']//p").all().get(2));
						if(showingDate.lastIndexOf("上线")!=-1){
							showingDate = showingDate.substring(0,showingDate.indexOf("上线"));
						}
						showingDate = showingDate.replaceAll("[\u4e00-\u9fa5-]", "");
						if(showingDate.length()==8){
							showingDate = convertDateStr(showingDate);
							dateAndArea[0] = showingDate;
						}
					}
				}
				if(null != showingArea && StringUtils.isNotBlank(showingArea)){
					dateAndArea = matchShowingAreaAndShowingDate(HtmlUtil.delHTMLTag(showingArea));
				}
				
			} catch (Exception e) {
				System.out.println(e.getMessage());
			}
			
			if(null == maoYanMovieBasicInfoModel.getEnglishName()){
				maoYanMovieBasicInfoModel.setEnglishName(englishName);
			}
			if(null == maoYanMovieBasicInfoModel.getLabel()){
				maoYanMovieBasicInfoModel.setLabel(label);
			}
			if(null == maoYanMovieBasicInfoModel.getArea()){
				maoYanMovieBasicInfoModel.setArea(area);
			}
			if(null == maoYanMovieBasicInfoModel.getTimeLength()){
				maoYanMovieBasicInfoModel.setTimeLength(timeLength);
			}
			if(null == maoYanMovieBasicInfoModel.getPlayType()){
				maoYanMovieBasicInfoModel.setPlayType(playType);
			}
			if(null == maoYanMovieBasicInfoModel.getShowingDate()){
				maoYanMovieBasicInfoModel.setShowingDate(dateAndArea[0]);
			}
			if(null == maoYanMovieBasicInfoModel.getShowingArea()){
				maoYanMovieBasicInfoModel.setShowingArea(dateAndArea[1]);
			}
			extrasMap.put("movieBasicInfo", maoYanMovieBasicInfoModel);
			PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/movie/%s/companylist", dataId), extrasMap);
		}
		
	}


	//基本信息
	@SuppressWarnings("unchecked")
	public void basicinfoProcess(Page page,Map<String, Object> extrasMap){
		
		String json = page.getRawText();
		if (StringUtils.isEmpty(json))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		Map<String, Object> map = GsonUtils.fromJson(json, Map.class);
		if (map == null || map.isEmpty() || map.get("sectionHTMLs") == null)
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		Map<String, Object> dataMap = ((Map<String, Object>) map.get("sectionHTMLs"));
		//数据id
		String dataId = extrasMap.get("dataId").toString();
		//电影名称
		String movieName = extrasMap.get("movieName").toString();
		//影片简介
		Map<String, Object> detailSectionMap = (Map<String, Object>) dataMap.get("detailSection");
		String introduction = new Html(detailSectionMap.get("html")==null?"":detailSectionMap.get("html").toString()).xpath("//section[@class='detail-section']//div[@class='detail-block-content']/text()").get();
		
		MaoYanMovieBasicInfoModel maoYanMovieBasicInfoModel = new MaoYanMovieBasicInfoModel(
				UUIDUtils.getId(),
				dataId,
				null,
				movieName,
				null,
				null,
				null,
				null,
				null,
				null,
				null,
				null,
				null,
				null,
				null,
				null,
				null,
				introduction,
				null,
				null,
				null,
				null,
				DateUtils.getDateString(new Date(), DateUtils
			            .SECONDS_FORMAT),
				null,
				null,
				null,
				null,
				null,
				null,
				null,null,null,null,null,null,null,null,null,null,null,null,null,
				null,null,resourceSiteId,null
				);
		extrasMap.put("movieBasicInfo", maoYanMovieBasicInfoModel);
		PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/movie/%s?_v_=yes", dataId), extrasMap);
	}
	
	
	public void addRequestProcess(Page page){
		
		String url = page.getRequest().getUrl();
		
		String json = page.getRawText();
        if (StringUtils.isEmpty(json))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        Map<String, Object> map = GsonUtils.fromJson(json, Map.class);
        if (map == null || map.isEmpty() || map.get("data") == null)
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        @SuppressWarnings("unchecked")
		Map<String, Object> dataMap = ((Map<String, Object>) map.get("data"));
        @SuppressWarnings("unchecked")
		List<Map<String, Object>> movList = (List<Map<String, Object>>) dataMap.get("list");
        if (movList == null || movList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        if(!StringUtils.isEmpty(url)){
			String dateTime = url.substring(url.lastIndexOf("=")+1);
			String key = "";
			if(dateTime.compareTo(todayDate)==0){//今天
				key = todayDate+"_MAOYAN_DATALIST_"+DateUtils.fomartDateToString(new Date(), "yyyyMMdd");
			}else if(dateTime.compareTo(todayDate)<0){//昨天
				key = yesterdayDate+"_MAOYAN_DATALIST_"+DateUtils.fomartDateToString(new Date(), "yyyyMMdd");
			}
        	cacheHandler.hSet(key, "MAOYAN_DATALIST", movList);
        	cacheHandler.setCacheTimeout(key);
		}
        movList.forEach(movMap ->
        {
            String dataId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, StringUtil
                    .nullToString(movMap.get("movieId")));
            if (StringUtils.isEmpty(dataId))
                return;
            Map<String, Object> paramMap = new HashMap<String, Object>();
            paramMap.put("dataId", dataId);
            paramMap.put("maoy_dataId", dataId);
            paramMap.put("movieName", StringUtil.nullToString(movMap.get("movieName")));
            paramMap.put("filedName", "maoy_id");
            paramMap.put("resourceSiteId", Constants.SiteName.MAOYAN);
            MaoYanMovieBasicInfoModel movieBasicInfo = maoYanMovieBasicInfoService.queryMovieBasicInfoList(paramMap);
			if(null == movieBasicInfo){
				PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/movie/%s/moresections", dataId), paramMap);
				//首先判断之前是否存在记录
				TVDataSourceRelatetionModel model = tvDataSourceRelationService.queryTvDataSourceRelation(paramMap);
				if(model == null){
					//插入剧数据源关系表中
					new AsyncProcess(new TVDataSourceRelatetionModel(
							null,//基础库剧id暂时没有
							null,
							null,
							dataId,
							StringUtil.nullToString(movMap.get("movieName")),
							null,
							null,
							System.currentTimeMillis()/1000
							)).asyncMethod();
				}
			}
        });
	}
	
	
	//临时测试用
//	public void addRequestProcess(Page page){
//        Map<String, Object> paramMap = new HashMap<String, Object>();
//        paramMap.put("dataId", "1212");
//        paramMap.put("movieName", StringUtil.nullToString("千与千寻"));
//        paramMap.put("filedName", "maoy_id");
//        paramMap.put("resourceSiteId", Constants.SiteName.MAOYAN);
//		PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/movie/%s/moresections", "1212"), paramMap);
//	}
	
	
	
	@Override
	protected String[] getStartRequestUrls() {
		return new String[]{String.format(LIST_URL[0],this.yesterdayDate),
				String.format(LIST_URL[1],this.todayDate)};
	}
	
	//临时测试
//	@Override
//	protected String[] getStartRequestUrls() {
//		return new String[]{String.format(LIST_URL[0],this.yesterdayDate)};
//	}

	@Override
	protected String getSpiderType() {
		return Constants.SpiderTaskType.MOVIE_BOXOFFICE;
	}

	@Override
	public String getPageDataType(Page page) {
		String url = page.getUrl().toString();
        if (url.contains("moresections")){
        	return Constants.PageDataType.BASICINFO;
        }else if (url.contains("_v_=yes")){
        	return "_v_=yes";
        }else if (url.contains("/companylist")){
        	return "/companylist";
        }else if (url.contains("/celebritylist")){
			return "/celebritylist";
		}else if(url.contains("/celebrity?id")){
			return "/celebrity?id";
		}else if(url.contains("/companydetail")){
			String tempUrl = url.substring(url.lastIndexOf("/")+1).replaceAll("[0-9]","");
			if(tempUrl.equals("")){
				return "/companydetail";
			}else if(tempUrl.equals("artist")){
				return "/artist";
			}
		}
        return Constants.PageDataType.SEARCH_LIST;
	}

	
	
	private String[] matchShowingAreaAndShowingDate(String str){
		
		String[] data = new String[2];
		if(StringUtils.isNotBlank(str)){
			if(regexStr("[\u4e00-\u9fa5-:]",str).length()>=8){
				data[0] = regexStr("[\u4e00-\u9fa5]",str);
				data[1] = regexStr("[0-9:-]",str);
			}else if (StringUtils.isEmpty(regexStr("[\u4e00-\u9fa5]",str))){
				data[1] = str.trim();
				data[0] = null;
			}else if (StringUtils.isEmpty(regexStr("[0-9:-]",str))){
				data[0] = str.trim();
				data[1] = null;
			}
		}
		
		return data;
	}
	
	
	private String regexStr(String regex,String str){
		
		String reg = regex;

		Pattern pat = Pattern.compile(reg);  

		Matcher mat=pat.matcher(str); 

		String returnStr = mat.replaceAll("");
		
		return returnStr;
	}
	
	protected String convertDateStr(String str){
    	String first = str.substring(0,4);
    	String second = str.substring(4, 6);
    	String three = str.substring(6);
    	return first+"-"+second+"-"+three;
    }
	
	@Autowired
	private TVDataSourceRelationService tvDataSourceRelationService;
	
	protected class AsyncProcess {
		
		protected TVDataSourceRelatetionModel obj;
		
		public AsyncProcess(TVDataSourceRelatetionModel obj){
			this.obj = obj;
		}
		
		@SuppressWarnings("unused")
		public void asyncMethod(){
			int availableProcessors = Runtime.getRuntime().availableProcessors();
			ExecutorService pool = Executors.newFixedThreadPool(availableProcessors);
			try {
				Integer fu = (Integer)pool.submit(new Callable<Integer>(){

					@Override
					public Integer call() throws Exception {
						
						return AsyncProcess.this.saveTVDataSourceRelation(AsyncProcess.this.obj);
					}
					
				}).get();
			} catch (InterruptedException e) {
				e.printStackTrace();
			} catch (ExecutionException e) {
				e.printStackTrace();
			}
			pool.shutdown();
		}
		
		
		@SuppressWarnings("unused")
		public void asyncUpdMethod(){
			int availableProcessors = Runtime.getRuntime().availableProcessors();
			ExecutorService pool = Executors.newFixedThreadPool(availableProcessors);
			try {
				Integer fu = (Integer)pool.submit(new Callable<Integer>(){

					@Override
					public Integer call() throws Exception {
						
						return AsyncProcess.this.updateTVDataSourceRelation(AsyncProcess.this.obj);
					}
					
				}).get();
			} catch (InterruptedException e) {
				e.printStackTrace();
			} catch (ExecutionException e) {
				e.printStackTrace();
			}
			pool.shutdown();
		}
		
		
		private int saveTVDataSourceRelation(TVDataSourceRelatetionModel obj){
			return tvDataSourceRelationService.saveTVDataSourceRelation(obj);
		}
	
		private int updateTVDataSourceRelation(TVDataSourceRelatetionModel obj){
			return tvDataSourceRelationService.updateTVDataSourceRelation(obj);
		}
		
		
	}
	
}
